package crawler.web.crawlDemo;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;

import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;

public class RetrivePage {
	private static HttpClient httpClient = new HttpClient();
	// 设置代理服务器
	static {
		// 设置代理服务器的IP地址和端口
		//		httpClient.getHostConfiguration().setProxy("113.209.149.186", 9001);
		httpClient.getHostConfiguration().setProxy("113.209.148.221", 9001);
		//		httpClient.getHostConfiguration().setProxy("113.209.129.100", 9001);
		//		httpClient.getHostConfiguration().setProxy("113.209.139.108", 9001);
		//		httpClient.getHostConfiguration().setProxy("113.209.144.17", 9001);
		//		httpClient.getHostConfiguration().setProxy("113.209.146.172", 9001);
		//		httpClient.getHostConfiguration().setProxy("113.209.139.46", 9001);
		//		httpClient.getHostConfiguration().setProxy("113.209.139.236", 9001);
		//		httpClient.getHostConfiguration().setProxy("113.209.151.59", 9001);
		//		httpClient.getHostConfiguration().setProxy("127.0.0.1", 8000);
	}

	public static String downloadPage(String urlstr) throws HttpException, IOException {
		StringBuilder sb = new StringBuilder();
		// 得到post方法
		GetMethod postMethod = new GetMethod(urlstr);
		// 设置post方法的参数
		/*
		 * NameValuePair[] postData = new NameValuePair[2]; postData[0] = new
		 * NameValuePair("name","lietu"); postData[1] = new
		 * NameValuePair("password","*****");
		 * postMethod.addParameters(postData);
		 */
		// 执行，返回状态码
		int statusCode = httpClient.executeMethod(postMethod);
		// 针对状态码进行处理 (简单起见，只处理返回值为200的状态码)
		if (statusCode == HttpStatus.SC_OK) {
			save2StringBuilder(sb, postMethod);
		}
		//若需要转向，则进行转向操作
		else if ((statusCode == HttpStatus.SC_MOVED_TEMPORARILY) || (statusCode == HttpStatus.SC_MOVED_PERMANENTLY) || (statusCode == HttpStatus.SC_SEE_OTHER) || (statusCode == HttpStatus.SC_TEMPORARY_REDIRECT)) {
			//读取新的URL地址
			Header header = postMethod.getResponseHeader("location");
			if (header != null) {
				String newUrl = header.getValue();
				if (newUrl == null || newUrl.equals("")) {
					newUrl = "/";
					//使用post转向
					PostMethod redirect = new PostMethod(newUrl);
					statusCode = httpClient.executeMethod(redirect);
					//发送请求，做进一步处理...
					//					save2StringBuilder(sb, redirect);
					//					redirect.releaseConnection();
				}
			}
		}
		return sb.toString();
	}

	private static void save2StringBuilder(StringBuilder sb, GetMethod postMethod) throws IOException {
		BufferedReader reader = new BufferedReader(new InputStreamReader(postMethod.getResponseBodyAsStream()));
		for (String line; (line = reader.readLine()) != null;) {
			sb.append(line);
		}
		if (reader != null) {
			reader.close();
		}
		//得到文件名
		//			String filename = urlstr.substring(urlstr.lastIndexOf('/') + 1);
		//获得文件输出流
		//			output = new FileOutputStream(filename);
		//输出到文件
		//			int tempByte = -1;
		//			while ((tempByte = input.read()) > 0) {
		//				output.write(tempByte);
		//			}
		//关闭输入输出流
	}

	/**
	 * 测试代码
	 */
	public static void main(String[] args) {
		try {
			String str = RetrivePage.downloadPage("http://www.baidu.com/");
			System.out.println("---->" + str + "<><><><>");
		} catch (HttpException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
		System.out.println("done");
	}
}
